{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b471e7c0",
   "metadata": {},
   "source": [
    "# ChatGPT Hate Speech Classification I (using main definition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dbbe5b37",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#pip install openai\n",
    "#!conda install --yes -c anaconda pandas\n",
    "#!conda install --yes -c anaconda scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "496986d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from openai import OpenAI\n",
    "import time\n",
    "import re\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e9087aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "notebook_path = os.path.abspath(\"__file__\")\n",
    "notebook_dir = os.path.dirname(notebook_path)\n",
    "print(notebook_dir)\n",
    "\n",
    "path='/Users/Mael/Library/CloudStorage/Dropbox/projects_active/eth_uzh_hatespeech_llm_finetuning/data/'\n",
    "path2='/Users/Mael/Library/CloudStorage/Dropbox/projects_active/eth_uzh_hatespeech_llm_finetuning/annotations/chatGPT/input_zero_shot/'\n",
    "path3='/Users/Mael/Library/CloudStorage/Dropbox/projects_active/eth_uzh_hatespeech_llm_finetuning/instructions_gpt_40/'\n",
    "path4='/Users/Mael/Library/CloudStorage/Dropbox/projects_active/eth_uzh_hatespeech_llm_finetuning/annotations/chatGPT/output_zero_shot/'\n",
    "\n",
    "df=pd.read_csv(path+'citizen_science_for_annots_27012022.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f78cdd98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 5)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df['check_hs']=df['check_hs'].astype(int)\n",
    "\n",
    "my_text=df['Kommentar'].tolist()\n",
    "\n",
    "df.to_csv(path2 +'chatGPT_data_readtouse.csv')\n",
    "\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8ccff421",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Anleitung:\\n',\n",
       " '\\n',\n",
       " 'Sie werden Kommentare zu Artikeln aus dem Internet lesen und bestimmen, ob es sich um Hate Speech handelt oder nicht. \\n',\n",
       " 'Es gibt keine allgemeingültige Definition von Hate Speech. Eine weit verbreitete – auch von den Vereinten Nationen verwendete – Definition versteht unter Hate Speech den Gebrauch von Sprache, durch den eine Person oder eine Gruppe aufgrund eines Merkmals oder einer Eigenschaft angegriffen oder abgewertet wird. Merkmale und Eigenschaften umfassen dabei nicht nur fest definierte Kategorien, sondern können sich auf jegliche Eigenschaften beziehen, anhand derer Menschen charakterisiert oder beschrieben werden – beispielsweise Geschlecht, Alter, Sexualität, Religion, Nationalität, Hautfarbe, Herkunft, geistige oder körperliche Beeinträchtigung, sozialer Status, politische Einstellung, Aussehen oder andere relevante Merkmale.\\n',\n",
       " 'Zusätzlich gibt es toxischen/hasserfüllten Sprachgebrauch (Toxic Speech), der zwar nicht notwendigerweise Hate Speech im engeren Sinne ist, aber die Diskussionskultur beeinträchtigt und Feindseligkeit fördert. Toxic Speech umfasst unsachliche, beleidigende, bedrohliche oder vulgäre Äusserungen, die aggressiv  wirken, ohne sich gezielt auf Merkmale oder Eigenschaften einer Person oder Gruppe zu beziehen.\\n',\n",
       " '\\n',\n",
       " 'Schritte\\n',\n",
       " 'Schritt 1: \\n',\n",
       " 'Lesen Sie den Kommentar. \\n',\n",
       " '\\n',\n",
       " 'Schritt 2:\\n',\n",
       " \"Wenn ein Kommentar KEINE HATE SPEECH enthält, beantworten Sie Frage 1 bitte mit 'False'. Wenn ein Kommentar HATE SPEECH oder TOXIC SPEECH enthält, beantworten Sie die Frage 1 bitte mit 'True'.\\n\",\n",
       " '\\n',\n",
       " 'Schritt 3: \\n',\n",
       " 'Nur wenn der Kommentar HATE SPEECH oder TOXIC SPEECH enthält, beantworten Sie bitte auch Frage 2, in der Sie angeben, gegen welche Personen-Gruppe(n) sich der Kommentar richtet oder im falle von toxischer Sprache wählen sie Toxisch.\\n',\n",
       " '\\n',\n",
       " 'Frage 1 (Label): \\n',\n",
       " 'Klassifiziere die Kommentare folgendermassen: \\n',\n",
       " 'True - Der Kommentar enthält Hate Speech oder Toxic Speech\\n',\n",
       " 'False - Der Kommentar enthält keine Hate Speech und auch keine Toxic Speech\\n',\n",
       " '\\n',\n",
       " 'Frage 2 (Target):\\n',\n",
       " 'Falls der Kommentar Hate Speech oder Toxische Sprache enthält gib an gegen welche Zielgruppe sich die Hatespeech richtet:\\n',\n",
       " 'Geschlecht - Die Hate Speech richtet sich gegen das Geschlecht\\n',\n",
       " 'Alter - Die Hate Speech richtet sich gegen das Alter\\n',\n",
       " 'Sexualität - Die Hate Speech richtet sich gegen die Sexualität\\n',\n",
       " 'Religion - Die Hate Speech richtet sich gegen die Religion\\n',\n",
       " 'Nationalität - Die Hate Speech richtet sich gegen die Nationalität oder  Hautfarbe oder Herkunft\\n',\n",
       " 'Beeinträchtigung - Die Hate Speech richtet sich gegen die geistliche oder körperliche Beeinträchtigung\\n',\n",
       " 'Toxisch - Richtet sich gegen keine Gruppe, benutzt aber toxische Sprache\\n',\n",
       " 'Status - Die Hate Speech richtet sich gegen den sozialen Status, Bildung, Einkomme oder Berufsgruppe\\n',\n",
       " 'Politik - Die Hate Speech richtet sich gegen die politische Einstellung\\n',\n",
       " 'Aussehen - Die Hate Speech richtet sich gegen das Aussehen oder körperliche Merkmale\\n',\n",
       " 'Andere - Die Hate Speech richtet sich gegen etwas anderes\\n',\n",
       " '\\n',\n",
       " 'Gib das Ergebnis immer im JSON-Format an, gib nichts anderes an. Hier ist ein Beispiel für das Format, das du verwenden musst:\\n',\n",
       " '\\n',\n",
       " '{\"Label\": \"<LABEL>\", \"Target\": \"<TARGET>\", \"reason\": \"Reasoning or evidence for the <LABEL> and <TARGET> chosen.\"}']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(path3+'/Instructions_main_definition.txt') as f:\n",
    "    instruction = f.readlines()\n",
    "instruction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "668b3bc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_key = '' # Your OpenAI API Key\n",
    "my_key_2 = '' # Your OpenAI API Key\n",
    "my_key_3 = '' # Your OpenAI API Key\n",
    "\n",
    "\n",
    "client = OpenAI(\n",
    "    api_key=my_key_3,  # This is the default and can be omitted\n",
    ")\n",
    "\n",
    "def run_chatgpt_default(instruction,my_text,k):\n",
    "\n",
    "    label=[]\n",
    "    text_list=[]\n",
    "\n",
    "    content=str(instruction)\n",
    "\n",
    "    for Kommentar in my_text:\n",
    "        i=my_text.index(Kommentar)\n",
    "        print(i)\n",
    "\n",
    "        completion = client.openai.chat.completions.create(\n",
    "          model=\"gpt-4o-mini-2024-07-18\",\n",
    "\n",
    "        messages=[\n",
    "          {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": content\n",
    "          },\n",
    "          {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": \"Here's the comment I picked, please label it as instructed as hate speech / toxic speech, or not and if there is hate speech / toxic speech also return the target group: \"+ Kommentar        \n",
    "\n",
    "          }\n",
    "        ]\n",
    "        )\n",
    "        label.append((completion.choices[0].message.content))\n",
    "        text_list.append(text)\n",
    "        print(completion.choices[0].message.content)\n",
    "\n",
    "\n",
    "    df_label=pd.DataFrame()\n",
    "    df_label['Kommentar']=text_list\n",
    "    df_label['label']=label\n",
    "\n",
    "    df_label.to_csv(path4+'label_hatespeech_round_main_definition_{0}.csv'.format(k))\n",
    "    return df_label\n",
    "\n",
    "def run_chatgpt_default(instruction, my_text, k, retry_limit=5):\n",
    "\n",
    "    label = []\n",
    "    text_list = []\n",
    "    content = str(instruction)\n",
    "\n",
    "    for Kommentar in my_text:\n",
    "        i = my_text.index(Kommentar)\n",
    "        print(i)\n",
    "\n",
    "        retry_count = 0\n",
    "        while True:\n",
    "            try:\n",
    "                completion = client.chat.completions.create(\n",
    "                    model=\"gpt-4o-mini-2024-07-18\",\n",
    "\n",
    "                    messages=[\n",
    "                        {\n",
    "                            \"role\": \"system\", \n",
    "                            \"content\": content\n",
    "                        },\n",
    "                        {\n",
    "                            \"role\": \"user\",\n",
    "                            \"content\": \"Here's the comment I picked, please label it as instructed as hate speech / toxic speech, or not and if there is hate speech / toxic speech also return the target group: \"+ Kommentar   \n",
    "\n",
    "                        }\n",
    "                    ]\n",
    "                )\n",
    "                break\n",
    "            except openai.RateLimitError as e:\n",
    "                retry_count += 1\n",
    "                if retry_count > retry_limit:\n",
    "                    raise e\n",
    "                wait_time = 60  # seconds\n",
    "                print(\n",
    "                    f\"Rate limit exceeded. Retrying in {wait_time} seconds... (Retry count: {retry_count})\"\n",
    "                )\n",
    "                time.sleep(wait_time)\n",
    "\n",
    "        label.append(completion.choices[0].message.content)\n",
    "        text_list.append(Kommentar)\n",
    "        print(completion.choices[0].message.content)\n",
    "\n",
    "    df_label = pd.DataFrame({\"Kommentar\": text_list, \"label\": label})\n",
    "    df_label.to_csv(path4 + f\"label_hatespeech_round_main_definition_{k}.csv\")\n",
    "\n",
    "    return df_label\n",
    "\n",
    "\n",
    "def evaluate(df_label):\n",
    "\n",
    "    df=pd.read_csv(path+'hatespeech_deciles_readytouse.csv')\n",
    "\n",
    "    print(df.shape)\n",
    "\n",
    "    df_final1=pd.concat([df,df_label],1)\n",
    "\n",
    "\n",
    "    df_final1=df_final1.dropna(subset='label')\n",
    "    df_final1['label']=df_final1['label'].apply(lambda x:find_label(x))\n",
    "\n",
    "    print('****************************\\n ChatGPT labelling value count:')\n",
    "    print('label: \\n',df_final1['label'].value_counts())\n",
    "\n",
    "\n",
    "    print('****************************\\n intersection with RA labelling:')\n",
    "    print(df_final1['intersection'].value_counts(normalize=True))\n",
    "\n",
    "\n",
    "    y_test=df_final1['check_hs']\n",
    "    prediction=df_final1['label']\n",
    "\n",
    "    print(confusion_matrix(y_test, prediction))\n",
    "    print(classification_report(y_test, prediction))\n",
    "\n",
    "\n",
    "    return df_final1\n",
    "\n",
    "\n",
    "\n",
    "def compare(df1,df2,l1,l2):\n",
    "    df_compare=pd.concat([df1,df2],1)\n",
    "\n",
    "    df_compare['label_intersection']=df_compare.apply(lambda x: find_intersections(x[l1],x[l2]),axis=1)\n",
    "\n",
    "    return (df_compare['label_intersection'].value_counts(normalize=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a67764b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('\\n Task 1: Round 1 with default temperature:')\n",
    "df_label1=run_chatgpt_default(instruction,my_text,1)\n",
    "#df_final1=evaluate(df_label1)\n",
    "#df_final1=df_final1.rename(columns={'label':'label1'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "095be38a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('\\n Task 2: Round 1 with default temperature:')\n",
    "df_label1=run_chatgpt_default(instruction,my_text,2)\n",
    "#df_final1=evaluate(df_label1)\n",
    "#df_final1=df_final1.rename(columns={'label':'label1'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7f88ccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('\\n Task 3: Round 1 with default temperature:')\n",
    "df_label1=run_chatgpt_default(instruction,my_text,3)\n",
    "#df_final1=evaluate(df_label1)\n",
    "#df_final1=df_final1.rename(columns={'label':'label1'})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatGPT",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
